# -*- coding: utf-8 -*-
# Author   : ZhangQing
# Time     : 2025-01-19 22:49
# File     : download_files_spider.py
# Project  : crawl-annual-reports
# Desc     :

import scrapy
import json
from scrapy.http import Request
from CrawlAnnualReports.items import CrawlannualreportsItem

class DownloadFilesSpider(scrapy.Spider):
    name = "download_files"

    def start_requests(self):
        with open("file_urls.json", "r", encoding="utf-8") as file:
            urls = json.load(file)
            for entry in urls:
                yield Request(url=entry["adjunct_url"], meta={"file_name": entry["file_name"]})

    def parse(self, response, **kwargs):
        file_name = response.meta["file_name"]
        with open(f"download/{file_name}.pdf", "wb") as f:
            f.write(response.content)
        print(f"{file_name}下载完毕")
